import re
import sys
import json
import argparse
import requests
from queue import Queue
from random import choice
from hashlib import md5
from threading import Thread, Lock
from time import strftime, localtime, sleep
from requests.packages.urllib3.exceptions import InsecureRequestWarning


# Check if we are running this on windows platform
is_windows = sys.platform.startswith('win')
# Console Colors
if is_windows:
    # Windows deserves coloring too :D
    G = '\033[92m'  # green
    Y = '\033[93m'  # yellow
    B = '\033[94m'  # blue
    R = '\033[91m'  # red
    W = '\033[0m'   # white
    try:
        import win_unicode_console
        import colorama
        win_unicode_console.enable()
        colorama.init()
        # Now the unicode will work ^_^
    except:
        print("[!] Error: Coloring libraries not installed, no coloring will be used [Check the readme]")
        G = Y = B = R = W = G = Y = B = R = W = ''
else:
    G = '\033[92m'  # green
    Y = '\033[93m'  # yellow
    B = '\033[94m'  # blue
    R = '\033[91m'  # red
    W = '\033[0m'   # white


# cancel color
def no_color():
    global G, Y, B, R, W
    G = Y = B = R = W = ''


def brand():
    print(f"""{R}
          _     _   _                                             
| |__ | |_| |_ _ __  ___  ___ __ _ _ __  _ __   ___ _ __ 
| '_ \| __| __| '_ \/ __|/ __/ _` | '_ \| '_ \ / _ \ '__|
| | | | |_| |_| |_) \__ \ (_| (_| | | | | | | |  __/ |   
|_| |_|\__|\__| .__/|___/\___\__,_|_| |_|_| |_|\___|_|   
              |_|                                       {W}v1.0{Y} 
         you can use -h for help
         """)
class Worker(Thread):  # 处理工作请求
    def __init__(self, work_queue, result_queue, **kwds):
        Thread.__init__(self, **kwds)
        # self.setDaemon(True)    # 设置为守护进程
        self.work_queue = work_queue
        self.result_queue = result_queue

    def run(self):
        callable, args, kwds = self.work_queue.get(False)
        callable(*args, **kwds)


class WorkManager:  # 线程池管理,创建
    def __init__(self, num_of_workers=8, time_wait=5):
        self.work_queue = Queue()  # 请求队列
        self.result_queue = Queue()  # 输出结果的队列
        self.workers = []
        self.time_wait = time_wait
        self.create_threads(num_of_workers)

    def create_threads(self, num_of_workers):

        for i in range(num_of_workers):
            worker = Worker(self.work_queue, self.result_queue)
            self.workers.append(worker)

    def start(self):
        for w in self.workers:
            w.start()

    def wait_for_complete(self):

        if self.time_wait < 0:
            for worker in self.workers:
                worker.join()
        else:
            for worker in self.workers:
                worker.join(self.time_wait)

    def add_job(self, callable, *args, **kwds):
        self.work_queue.put((callable, args, kwds))

    def get_result(self, *args, **kwds):
        return self.result_queue.get(*args, **kwds)


requests.packages.urllib3.disable_warnings(InsecureRequestWarning)  # 取消显示证书的警告信息
requests.adapters.DEFAULT_RETRIES = 5   # 增加重試次數
finger_db = json.load(open('./fingerprints.json'))


def get_md5(text):
    """
    生成md5值
    :param text: 字节数组
    :return: md5字符串
    """
    return md5(text).hexdigest()


def url_handle(url, path):
    """
    对目标url和路径合并
    :param url:
    :param path:
    :return:
    """
    if not url.startswith("http://") and not url.startswith("https://"):
        url = "http://" + url
    elif url[-1] == '/':
        url = url[:-1]
    if path == '' or path[0] != '/':
        path = '/' + path
    return url + path


def get_headers():
    """
    获取随机的http头
    :return: http头字典
    """
    li = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19582',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36 Edge/18.19577',
        'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:77.0) Gecko/20190101 Firefox/77.0',
        'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:77.0) Gecko/20100101 Firefox/77.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A',
        'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25',
        'Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
        'Mozilla/5.0 (X11; Ubuntu; Linux i686 on x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2820.59 Safari/537.36',
        'Mozilla/5.0 (X11; Linux ppc64le; rv:75.0) Gecko/20100101 Firefox/75.0',
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:75.0) Gecko/20100101 Firefox/75.0',
        'Mozilla/5.0 (X11; Linux; rv:74.0) Gecko/20100101 Firefox/74.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:61.0) Gecko/20100101 Firefox/73.0',
        'Mozilla/5.0 (compatible; U; ABrowse 0.6; Syllable) AppleWebKit/420+ (KHTML, like Gecko)',
        'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2',
        'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:69.2.1) Gecko/20100101 Firefox/69.2',
        'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.10; rv:62.0) Gecko/20100101 Firefox/62.0',
    ]

    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'User-Agent': choice(li),
        'Upgrade-Insecure-Requests': '1',
        'Connection': 'keep-alive',
        'Cache-Control': 'max-age=0',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'zh-CN,zh;q=0.8',
        "Referer": "http://www.baidu.com/?wd=test",
    }
    return headers


def find_version(keyword: str, target_text: str):
    match = re.findall(r"^.*?(" + keyword + r")[a-zA-Z0-9\-_\s]{0,35}?([0-9]+(\.[0-9]+)*).*$", target_text.lower(),
                       re.S | re.I)
    if match:
        for match_one in match:
            if match_one[1] != '':
                return match_one[1]
        return match[0][1]
    return False


class HttpScanner:

    def __init__(self, target_url, delay=0, timeout=3):
        url_parse = re.findall(r"(.*(//|^)([a-zA-Z0-9\-:]+(\.[a-zA-Z0-9\-:]+)*))(.*)", target_url)
        self.target_url = url_parse[0][0]
        self.path = url_parse[0][4]
        self.result = {"target": target_url}
        self.index = 0
        self.delay = delay
        self.lock = Lock()
        self.timeout = timeout
        self.finger_db = finger_db
        self.finger_count = {}

        if self.path != '' and self.path != '/':
            lenv = len(self.finger_db)
            self.finger_db.extend(self.finger_db)
            for i in range(lenv, 2 * lenv):
                self.finger_db[i][2] = ('/' + self.path + self.finger_db[i][2]).replace('//', '/')

        for fin in self.finger_db:
            if self.finger_count.get(fin[0]) is None:
                self.finger_count[fin[0]] = 1
            else:
                self.finger_count[fin[0]] += 1

    def answer_handler(self, finger_item, ans):
        """
        扫描结果数据加工
        :param finger_item:
        :param ans:
        :return:
        """
        if ans[0]:
            self.lock.acquire()
            if self.result.get(finger_item[0]) is not None:
                self.result[finger_item[0]][1] += 1 / self.finger_count[finger_item[0]]
                if ans[1] != '':
                    self.result[finger_item[0]][0].add(ans[1])
            else:
                if ans[1] == '':
                    self.result[finger_item[0]] = [set(), 1 / self.finger_count[finger_item[0]]]
                else:
                    self.result[finger_item[0]] = [{ans[1], }, 1 / self.finger_count[finger_item[0]]]
            self.lock.release()

    def get_res(self, url: str, allow_redirects: bool = True):
        """
        请求url 获取响应体
        :param allow_redirects:
        :param url:
        :return: respond
        """
        if not url.startswith("http://") and not url.startswith("https://"):
            url = "http://" + url
        elif url[-1] == '/':
            url = url[:-1]
        try:
            res = requests.get(url=url, headers=get_headers(), timeout=self.timeout, verify=False,
                               allow_redirects=allow_redirects)
            sleep(self.delay)
            return res
        except Exception as e:
            print(e)
            return None

    def md5_compare(self, url: str, correct_md5: str) -> list:
        """
        页面md5是否符和预期
        :param url: 请求的页面
        :param correct_md5: 预期的md5值
        :return:
        """
        res = self.get_res(url)
        result = [False, '']

        if res is not None and res.status_code == 200:
            if get_md5(res.content) == correct_md5:
                result[0] = True
        return result

    def keyword_compare(self, url: str, keyword: str) -> list:
        """
        页面是否存在指纹关键字
        :param url: 请求的页面
        :param keyword: 预期的关键字
        :return:
        """
        res = self.get_res(url)
        keyword = keyword.lower()
        result = [False, '']

        if res is not None:
            if keyword in res.text.lower():
                result[0] = True
                may_version1 = find_version(keyword, res.text)
                if may_version1:
                    result[1] = may_version1
        return result

    @staticmethod
    def header_reg(res, key, value):
        result = [False, '']
        if res is not None:
            for r_key, r_value in res.headers.items():
                if key in r_key.lower() and value in r_value.lower():
                    result[0] = True
                    if value.lower() != '':
                        may_version1 = find_version(value, r_value)
                        may_version2 = find_version(value + r"|.*?", r_key)
                        if may_version1:
                            result[1] = may_version1
                            return result
                        if may_version2:
                            result[1] = may_version2
                            return result
                    if key.lower() != '':
                        may_version1 = find_version(key + "|.*?", r_value)
                        may_version2 = find_version(key, r_key)
                        if may_version1:
                            result[1] = may_version1
                            return result
                        if may_version2:
                            result[1] = may_version2
                            return result
                    return result
        return result

    def header_compare(self, url: str, key: str = "", value: str = "") -> list:
        """
        http头是否存在预期信息
        :param url: 请求的页面
        :param key:
        :param value:
        :return:
        """
        res = self.get_res(url)
        key = key.lower()
        value = value.lower()
        result = HttpScanner.header_reg(res, key, value)
        if result[0]:
            return result
        elif self.path != '' and self.path != '/':
            res = self.get_res(url_handle(url, self.path))
            result = HttpScanner.header_reg(res, key, value)
        return result

    def dir_compare(self, url: str) -> list:
        """
        判斷目錄是否存在
        :param url:
        :return:
        """
        res = self.get_res(url, False)
        result = [False, '']
        if res is not None:
            if res.status_code == 200 and res.status_code == 405 and res.status_code == 403:
                result[0] = True
        return result

    def regex_compare(self, url: str, reg: str, keyword: str) -> list:
        res = self.get_res(url)
        result = [False, '']
        if res is not None:
            match = re.findall(reg, res.text, re.I | re.S)
            for match_one in match:
                if isinstance(match_one, str):
                    if keyword in match_one:
                        result[0] = True
                        reg_res = find_version(keyword, match_one)
                        if reg_res:
                            result[1] = reg_res
                            return result
                elif isinstance(match_one, tuple):
                    if keyword in match_one[0]:
                        result[0] = True
                        reg_res = find_version(keyword, match_one[0])
                        if reg_res:
                            result[1] = reg_res
                            return result
        return result

    def url_compare(self, url: str, keyword: str):
        """
        通過重定向記錄匹配url裏的指紋
        :param url:
        :param keyword:
        :return:
        """
        res = self.get_res(url)
        result = [False, '']
        keyword = keyword.lower()
        if res is not None:
            redit_list = res.history
            for redit_res in redit_list:
                if keyword in redit_res.headers['location'].lower():
                    result[0] = True
                    return result
        return result

    def find_finger(self):
        """
        任务主函数
        :return:
        """
        while True:
            if self.index >= len(self.finger_db):
                break
            self.lock.acquire()
            try:
                fin = self.finger_db[self.index]
            except Exception as e:
                break
            self.index += 1

            print("[*] 正在识别指纹", end='')
            print(f"({self.index}/{len(self.finger_db)}): ")
            print(f"[*] {fin}")
            self.lock.release()

            if fin[1] == 'md5':
                ans = self.md5_compare(url_handle(self.target_url, fin[2]), fin[3])
                self.answer_handler(fin, ans)
            elif fin[1] == 'keyword':
                ans = self.keyword_compare(url_handle(self.target_url, fin[2]), fin[3])
                self.answer_handler(fin, ans)
            elif fin[1] == 'header':
                ans = self.header_compare(url_handle(self.target_url, fin[2]), fin[3], fin[4])
                self.answer_handler(fin, ans)
            elif fin[1] == 'dir':
                ans = self.dir_compare(url_handle(self.target_url, fin[2]))
                self.answer_handler(fin, ans)
            elif fin[1] == 'regex':
                ans = self.regex_compare(url_handle(self.target_url, fin[2]), fin[3], fin[4])
                self.answer_handler(fin, ans)
            elif fin[1] == 'url':
                ans = self.url_compare(url_handle(self.target_url, fin[2]), fin[3])
                self.answer_handler(fin, ans)
            else:
                print("[!] 指纹类型暂不支持")


def scanning(callable, thread_num, wait=-1):
    """
    启动所有线程，开启扫描任务
    :param callable: 要执行的函数
    :param thread_num: 线程数量
    :param wait:
    :return:
    """
    wm_domain_task = WorkManager(thread_num, wait)
    for i in range(thread_num):
        wm_domain_task.add_job(callable)
    wm_domain_task.start()
    wm_domain_task.wait_for_complete()


if __name__ == '__main__':
    brand()
    # 获取命令行参数
    if len(sys.argv) == 1:
        sys.argv.append("-h")
    parser = argparse.ArgumentParser()
    parser.add_argument('-v', '--version', action='version',
                        version='%(prog)s version : v1.0', help='show the version')
    parser.add_argument('-u', '--url', dest='url', help='your scan target', type=str)
    parser.add_argument('-f', '--file', dest='urlfile', help='target list', type=str)
    parser.add_argument('-t', '--threads', dest='threads', help='thread number, default: 10', type=int, default=3)
    parser.add_argument('-d', '--delay', dest='delay', help='delay time after each request, default: 0', type=int,
                        default=0)
    parser.add_argument('-o', '--output', dest='output_prefix',
                        help='result save path prefix, default: "./output/result"', type=str, default='./output/result')
    parser.add_argument('-to', '--timeout', dest='timeout',
                        help='every request timeout, default: 3', type=int, default=3)
    args = parser.parse_args()

    # 解析命令行参数
    urls = [args.url]
    if args.urlfile is not None:
        urls = [i.strip() for i in open(args.urlfile, 'r').readlines()]
    threads = args.threads
    delay = args.delay
    output_path_prefix = args.output_prefix
    timeout = args.timeout
    results = []

    # 开始批量扫描url
    for url in urls:
        scanner_obj = HttpScanner(target_url=url, delay=delay, timeout=timeout)
        scanning(scanner_obj.find_finger, threads, -1)
        print(f"[+] {url}扫描完毕，扫描结果为：")
        for key in scanner_obj.result.keys():    # 为了存储为json好看点，处理下数据
            if key != "target":
                scanner_obj.result[key][0] = list(scanner_obj.result[key][0])
                scanner_obj.result[key][1] = f"{scanner_obj.result[key][1]:.2%}"
                print(f"[+] 指纹:{key:<20} 可能版本:{scanner_obj.result[key]!r}")
        results.append(scanner_obj.result)

    # 存储扫描结果
    print("[*] 开始存储本次的扫描结果")
    f_path = output_path_prefix + strftime("%Y%m%d%H%M%S", localtime()) + ".json"
    with open(f_path, 'w') as file_obj:
        json.dump(results, file_obj, indent=4)
    print(f"[+] 本次的扫描结果已保存在：{f_path}")
    print("[+] 任務已經全部完成 程序結束")
